#!/usr/bin/env ruby
# -*- coding: us-ascii -*-
# Elizabeth Wiethoff 2008

raise NotImplementedError, 'need version >= 1.8' if RUBY_VERSION < '1.8'

require 'enumerator'
require 'md5'
require 'set'
require 'yaml'

dir = File.dirname(__FILE__)
require File.join(dir, '..', 'auxil', 'htmlstripper')

########################################

module SarrayLibSkipWords
  skip_path = File.join(File.dirname(__FILE__), 'connectives.txt')
  COMMON_WORDS = File.open(skip_path) { |f| f.read }.split.to_set
  
  yaml_path = File.join(File.dirname(__FILE__), 'codewords.yaml')
  yaml = File.open(yaml_path) { |file| YAML.load(file) }
  
  LETTERS = yaml['letters'].to_set
  #HTML = HTM = yaml['html'].to_set
  DB = KP = yaml['db'].to_set
  RTF = yaml['rtf'].to_set
  
  # valid arguments are :db, :html, :rtf, someday others
  private
  def const_from(sym)
    SarrayLibSkipWords.const_get(sym.to_s.upcase)
  end
end

########################################

# monkey patching built-in

class String
  include SarrayLibSkipWords
  
  def munge(skip_common, *special_skips)
    a = downcase.tr('^a-z', ' ').split
    a.reject! { |word| COMMON_WORDS.include? word } if skip_common
    special_skips.each do |sym|
      a.reject! { |word| const_from(sym).include? word }
    end
    a
  end
end

########################################

class Document
  @@threshold = 20
  @@skip_common_words = true
  
  def Document.threshold=(numeric)
    @@threshold = numeric
  end
  def Document.threshold ; @@threshold ; end
  
  def Document.skip_common_words=(bool)
    @@skip_common_words = bool
  end
  def Document.skip_common_words ; @@skip_common_words ; end
  
  def Document.process(path, text)
    SarrayLibC::PATHS.store path
    if path =~ /\.html?$/i
      plain_text = HTMLStripper.all_text(text)
      arr = plain_text.munge(@@skip_common_words, :letters)
    elsif path =~ /\.mht$/i  # kludge
      sorta_plain = HTMLStripper.all_text(text)
      arr = sorta_plain.munge(@@skip_common_words, :letters)
    elsif path =~ /\.(db|kp)$/i
      sorta_plain = HTMLStripper.all_text(text)
      arr = sorta_plain.munge(@@skip_common_words, $1, :letters)
      if $DEBUG and rand(50).zero?
        p path
        p arr
      end
    elsif path =~ /\.rtf$/i
      arr = text.munge(@@skip_common_words, 'rtf', :letters)
    else
      arr = text.munge(@@skip_common_words, :letters)
    end
    arr.each_cons(@@threshold) do |a|
      SarrayLibC::DATA.add(a, path.hash)
    end
    Thread.pass  # allow GC to do its thing
  end
end

########################################

class Dupes < Array
  def Dupes.find_em
    dupes = Dupes.new
    
    if $DEBUG
      n, m = SarrayLibC::DATA.n, SarrayLibC::DATA.m
      puts "n, m: #{n}, #{m}"
      puts "load factor: #{n / Float(m)}"
    end
    
    SarrayLibC::DATA.all_groups! do |group|
      Thread.pass  # allow GC to do its thing
      next unless group.size > 1
      dupes << group
    end
    dupes.uniq!
    dupes.map! { |grp| grp.map! { |ph| SarrayLibC::PATHS[ph] }.sort! }
    dupes
  end
  
  alias :old_sort! :sort!
  def sort!
    old_sort! { |a,b| [a.size, a] <=> [b.size, b] }.reverse!
  end
  
  def sort
    d = self.dup
    d.sort!
  end
  
  def print(pll=50)
    self.each do |paths|
      path_ends = paths.map do |pa|
        pa.length < pll ? pa : pa[-pll, pll]
      end
      puts 'Group: ' << path_ends.inspect
      puts
    end
  end
end

########################################

class SarrayHash < Hash
  def [](k)
    # explicitly do Array#join in case Array#to_s has been hacked
    k = k.join('') if k.respond_to? :to_ary
    k = MD5.new(k).hexdigest.hex
    
    val = super(k)
    return val if val
    self[k] = Array.new
  end
  
  def add(k, val)
    unless k.respond_to? :to_str or k.respond_to? :to_ary
      raise TypeError, 'key must be String-like or Array-like'
    end
    
    self[k] << val
    val
  end
  
  def m
    size
  end
  
  def n
    values.inject(0) { |tot, buck| tot += buck.size }
  end
  
  def all_groups!(&block)
    groups = values
    self.clear
    yield groups.pop until groups.empty?
  end
end

########################################

module SarrayLibC
  if $full_hash
    DATA = SarrayHash.new
  else
    currdir = File.dirname(__FILE__)
    require File.join(currdir, '..', 'src', 'lib', 'auxil', 'simplehash')
    DATA = SimpleGroups.new(49999)  # prime
    puts '*** SimpleGroups ***'
  end
  PATHS = Hash.new
  
  def PATHS.store(path)
    self[path.hash] = path
  end
end

def toss_garbage
  SarrayLibC::DATA.clear  # okay if it's already empty
  SarrayLibC::PATHS.clear
end

########################################

if $0 == __FILE__
  Document.threshold = 2
  Document.skip_common_words = false
  
  Document.process('getty.txt', 'Four score and seven years ago...')
  Document.process('yoda.txt', 'Score, and seven years you spend.')
  Document.process('panda.txt', 
    "You'll love the years you spend with pandas!")
  Document.process('six.txt', 
    'Twenty-six and seven years make thirty-three.')
  Document.process('bushel.txt', 
    'I love you a bushel and a peck, a bushel and a peck I do.')
  
  dupes = Dupes.find_em
  toss_garbage
  
  dupes.sort.print
  puts $/ + 'FINIS!'
end
